package org.zjt.hdfs.demo;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.BufferedWriter;
import java.io.File;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.net.URI;

/**
 * 设置fs.defaultFS参数，如果没有设置，会出现java.lang.IllegalArgumentException:
 * Wrong FS:hdfs://master:8020/user,expected: file:///
 * 当然了，也可以将hadoop集群中的core-site.xml配置文件拷贝到该工程项目下,这样，在读取配置文件时就能够识别hdfs文件系统
 * <p>
 * hdfs : file append
 *
 * /Users/zhangjuntao/Downloads/catalina.log
 *
 *
 * 172.19.110.240 master
 * 172.19.110.243 slave1
 * 172.19.110.244 slave2
 *
 */
public class HDFSUpdate {
    public static void main(String[] args) throws Exception {
        try {
            Configuration conf = new Configuration();// 读取配置文件
            conf.set("mapred.jop.tracker", "172.19.110.240:9001");
            conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
            conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
            conf.set("dfs.support.append", "true");


            
            FileSystem fs = FileSystem.get(new URI("hdfs://172.19.110.240:8020/"), conf, "root");// 获取hdfs实例
            Path dest = new Path("hdfs://172.19.110.240:8020/user/catalina.log");

            if (!fs.exists(new Path("hdfs://172.19.110.240:8020/user/"))) {
                fs.mkdirs(new Path("hdfs://172.19.110.240:8020/user/"));
            }
            FSDataOutputStream fos;
            if (!fs.exists(dest)) {
                fos = fs.create(dest, true);
            } else {
                fos = fs.append(dest);
            }
            BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fos));
            // TO append data to a file, use fs.append(Path f)
            InputStream is = FileUtils.openInputStream(new File("/Users/zhangjuntao/Downloads/catalina.log"));
            byte[] buffer = new byte[1024];
            int size ;
            for (;(size = is.read(buffer)) > 0 ;){
                br.write(new String(buffer, 0, size));
            }
            br.close();
            fs.close();
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("File not found");

        }
    }
}
